{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Implementing an LSTM RNN Model\n",
    "------------------------\n",
    "Here we implement an LSTM model on all a data set of Shakespeare works.\n",
    "\n",
    "We start by loading the necessary libraries and resetting the default computational graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import string\n",
    "import requests\n",
    "import numpy as np\n",
    "import collections\n",
    "import random\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import ops\n",
    "ops.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We start a computational graph session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next, it is important to set the algorithm and data processing parameters.\n",
    "\n",
    "---------\n",
    "Parameter  :  Descriptions\n",
    " - min_word_freq: Only attempt to model words that appear at least 5 times.\n",
    " - rnn_size: size of our RNN (equal to the embedding size)\n",
    " - epochs: Number of epochs to cycle through the data\n",
    " - batch_size: How many examples to train on at once\n",
    " - learning_rate: The learning rate or the convergence paramter\n",
    " - training_seq_len: The length of the surrounding word group (e.g. 10 = 5 on each side)\n",
    " - embedding_size: Must be equal to the rnn_size\n",
    " - save_every: How often to save the model\n",
    " - eval_every: How often to evaluate the model\n",
    " - prime_texts: List of test sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Set RNN Parameters\n",
    "min_word_freq = 5 # Trim the less frequent words off\n",
    "rnn_size = 128 # RNN Model size\n",
    "embedding_size = 100 # Word embedding size\n",
    "epochs = 10 # Number of epochs to cycle through data\n",
    "batch_size = 100 # Train on this many examples at once\n",
    "learning_rate = 0.001 # Learning rate\n",
    "training_seq_len = 50 # how long of a word group to consider \n",
    "embedding_size = rnn_size\n",
    "save_every = 500 # How often to save model checkpoints\n",
    "eval_every = 50 # How often to evaluate the test sentences\n",
    "prime_texts = ['thou art more', 'to be or not to', 'wherefore art thou']\n",
    "\n",
    "# Download/store Shakespeare data\n",
    "data_dir = 'temp'\n",
    "data_file = 'shakespeare.txt'\n",
    "model_path = 'shakespeare_model'\n",
    "full_model_dir = os.path.join(data_dir, model_path)\n",
    "\n",
    "# Declare punctuation to remove, everything except hyphens and apostrophes\n",
    "punctuation = string.punctuation\n",
    "punctuation = ''.join([x for x in punctuation if x not in ['-', \"'\"]])\n",
    "\n",
    "# Make Model Directory\n",
    "if not os.path.exists(full_model_dir):\n",
    "    os.makedirs(full_model_dir)\n",
    "\n",
    "# Make data directory\n",
    "if not os.path.exists(data_dir):\n",
    "    os.makedirs(data_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Download the data if we don't have it saved already.  The data comes from the [Gutenberg Project](http://www.gutenberg.org])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Shakespeare Data\n",
      "Cleaning Text\n",
      "Done loading/cleaning.\n"
     ]
    }
   ],
   "source": [
    "print('Loading Shakespeare Data')\n",
    "# Check if file is downloaded.\n",
    "if not os.path.isfile(os.path.join(data_dir, data_file)):\n",
    "    print('Not found, downloading Shakespeare texts from www.gutenberg.org')\n",
    "    shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt'\n",
    "    # Get Shakespeare text\n",
    "    response = requests.get(shakespeare_url)\n",
    "    shakespeare_file = response.content\n",
    "    # Decode binary into string\n",
    "    s_text = shakespeare_file.decode('utf-8')\n",
    "    # Drop first few descriptive paragraphs.\n",
    "    s_text = s_text[7675:]\n",
    "    # Remove newlines\n",
    "    s_text = s_text.replace('\\r\\n', '')\n",
    "    s_text = s_text.replace('\\n', '')\n",
    "    \n",
    "    # Write to file\n",
    "    with open(os.path.join(data_dir, data_file), 'w') as out_conn:\n",
    "        out_conn.write(s_text)\n",
    "else:\n",
    "    # If file has been saved, load from that file\n",
    "    with open(os.path.join(data_dir, data_file), 'r') as file_conn:\n",
    "        s_text = file_conn.read().replace('\\n', '')\n",
    "\n",
    "# Clean text\n",
    "print('Cleaning Text')\n",
    "s_text = re.sub(r'[{}]'.format(punctuation), ' ', s_text)\n",
    "s_text = re.sub('\\s+', ' ', s_text ).strip().lower()\n",
    "print('Done loading/cleaning.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define a function to build a word processing dictionary (word -> ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Build word vocabulary function\n",
    "def build_vocab(text, min_word_freq):\n",
    "    word_counts = collections.Counter(text.split(' '))\n",
    "    # limit word counts to those more frequent than cutoff\n",
    "    word_counts = {key:val for key, val in word_counts.items() if val>min_word_freq}\n",
    "    # Create vocab --> index mapping\n",
    "    words = word_counts.keys()\n",
    "    vocab_to_ix_dict = {key:(ix+1) for ix, key in enumerate(words)}\n",
    "    # Add unknown key --> 0 index\n",
    "    vocab_to_ix_dict['unknown']=0\n",
    "    # Create index --> vocab mapping\n",
    "    ix_to_vocab_dict = {val:key for key,val in vocab_to_ix_dict.items()}\n",
    "    \n",
    "    return(ix_to_vocab_dict, vocab_to_ix_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we can build the index-vocabulary from the Shakespeare data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building Shakespeare Vocab\n",
      "Vocabulary Length = 8009\n"
     ]
    }
   ],
   "source": [
    "# Build Shakespeare vocabulary\n",
    "print('Building Shakespeare Vocab')\n",
    "ix2vocab, vocab2ix = build_vocab(s_text, min_word_freq)\n",
    "vocab_size = len(ix2vocab) + 1\n",
    "print('Vocabulary Length = {}'.format(vocab_size))\n",
    "# Sanity Check\n",
    "assert(len(ix2vocab) == len(vocab2ix))\n",
    "\n",
    "# Convert text to word vectors\n",
    "s_text_words = s_text.split(' ')\n",
    "s_text_ix = []\n",
    "for ix, x in enumerate(s_text_words):\n",
    "    try:\n",
    "        s_text_ix.append(vocab2ix[x])\n",
    "    except:\n",
    "        s_text_ix.append(0)\n",
    "s_text_ix = np.array(s_text_ix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We define the LSTM model.  The methods of interest are the `__init__()` method, which defines all the model variables and operations, and the `sample()` method which takes in a sample word and loops through to generate text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Define LSTM RNN Model\n",
    "class LSTM_Model():\n",
    "    def __init__(self, embedding_size, rnn_size, batch_size, learning_rate,\n",
    "                 training_seq_len, vocab_size, infer_sample=False):\n",
    "        self.embedding_size = embedding_size\n",
    "        self.rnn_size = rnn_size\n",
    "        self.vocab_size = vocab_size\n",
    "        self.infer_sample = infer_sample\n",
    "        self.learning_rate = learning_rate\n",
    "        \n",
    "        if infer_sample:\n",
    "            self.batch_size = 1\n",
    "            self.training_seq_len = 1\n",
    "        else:\n",
    "            self.batch_size = batch_size\n",
    "            self.training_seq_len = training_seq_len\n",
    "        \n",
    "        self.lstm_cell = tf.contrib.rnn.BasicLSTMCell(self.rnn_size)\n",
    "        self.initial_state = self.lstm_cell.zero_state(self.batch_size, tf.float32)\n",
    "        \n",
    "        self.x_data = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])\n",
    "        self.y_output = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])\n",
    "        \n",
    "        with tf.variable_scope('lstm_vars'):\n",
    "            # Softmax Output Weights\n",
    "            W = tf.get_variable('W', [self.rnn_size, self.vocab_size], tf.float32, tf.random_normal_initializer())\n",
    "            b = tf.get_variable('b', [self.vocab_size], tf.float32, tf.constant_initializer(0.0))\n",
    "        \n",
    "            # Define Embedding\n",
    "            embedding_mat = tf.get_variable('embedding_mat', [self.vocab_size, self.embedding_size],\n",
    "                                            tf.float32, tf.random_normal_initializer())\n",
    "                                            \n",
    "            embedding_output = tf.nn.embedding_lookup(embedding_mat, self.x_data)\n",
    "            rnn_inputs = tf.split(axis=1, num_or_size_splits=self.training_seq_len, value=embedding_output)\n",
    "            rnn_inputs_trimmed = [tf.squeeze(x, [1]) for x in rnn_inputs]\n",
    "        \n",
    "        # If we are inferring (generating text), we add a 'loop' function\n",
    "        # Define how to get the i+1 th input from the i th output\n",
    "        def inferred_loop(prev, count):\n",
    "            # Apply hidden layer\n",
    "            prev_transformed = tf.matmul(prev, W) + b\n",
    "            # Get the index of the output (also don't run the gradient)\n",
    "            prev_symbol = tf.stop_gradient(tf.argmax(prev_transformed, 1))\n",
    "            # Get embedded vector\n",
    "            output = tf.nn.embedding_lookup(embedding_mat, prev_symbol)\n",
    "            return(output)\n",
    "        \n",
    "        decoder = tf.contrib.legacy_seq2seq.rnn_decoder\n",
    "        outputs, last_state = decoder(rnn_inputs_trimmed,\n",
    "                                      self.initial_state,\n",
    "                                      self.lstm_cell,\n",
    "                                      loop_function=inferred_loop if infer_sample else None)\n",
    "        # Non inferred outputs\n",
    "        output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.rnn_size])\n",
    "        # Logits and output\n",
    "        self.logit_output = tf.matmul(output, W) + b\n",
    "        self.model_output = tf.nn.softmax(self.logit_output)\n",
    "        \n",
    "        loss_fun = tf.contrib.legacy_seq2seq.sequence_loss_by_example\n",
    "        loss = loss_fun([self.logit_output],[tf.reshape(self.y_output, [-1])],\n",
    "                [tf.ones([self.batch_size * self.training_seq_len])],\n",
    "                self.vocab_size)\n",
    "        self.cost = tf.reduce_sum(loss) / (self.batch_size * self.training_seq_len)\n",
    "        self.final_state = last_state\n",
    "        gradients, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tf.trainable_variables()), 4.5)\n",
    "        optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "        self.train_op = optimizer.apply_gradients(zip(gradients, tf.trainable_variables()))\n",
    "        \n",
    "    def sample(self, sess, words=ix2vocab, vocab=vocab2ix, num=10, prime_text='thou art'):\n",
    "        state = sess.run(self.lstm_cell.zero_state(1, tf.float32))\n",
    "        word_list = prime_text.split()\n",
    "        for word in word_list[:-1]:\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[word]\n",
    "            feed_dict = {self.x_data: x, self.initial_state:state}\n",
    "            [state] = sess.run([self.final_state], feed_dict=feed_dict)\n",
    "\n",
    "        out_sentence = prime_text\n",
    "        word = word_list[-1]\n",
    "        for n in range(num):\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[word]\n",
    "            feed_dict = {self.x_data: x, self.initial_state:state}\n",
    "            [model_output, state] = sess.run([self.model_output, self.final_state], feed_dict=feed_dict)\n",
    "            sample = np.argmax(model_output[0])\n",
    "            if sample == 0:\n",
    "                break\n",
    "            word = words[sample]\n",
    "            out_sentence = out_sentence + ' ' + word\n",
    "        return(out_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "In order to use the same model (with the same trained variables), we need to share the variable scope between the trained model and the test model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Define LSTM Model\n",
    "lstm_model = LSTM_Model(embedding_size, rnn_size, batch_size, learning_rate,\n",
    "                        training_seq_len, vocab_size)\n",
    "\n",
    "# Tell TensorFlow we are reusing the scope for the testing\n",
    "with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
    "    test_lstm_model = LSTM_Model(embedding_size, rnn_size, batch_size, learning_rate,\n",
    "                                 training_seq_len, vocab_size, infer_sample=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We need to save the model, so we create a model saving operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create model saver\n",
    "saver = tf.train.Saver(tf.global_variables())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's calculate how many batches are needed for each epoch and split up the data accordingly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create batches for each epoch\n",
    "num_batches = int(len(s_text_ix)/(batch_size * training_seq_len)) + 1\n",
    "# Split up text indices into subarrays, of equal size\n",
    "batches = np.array_split(s_text_ix, num_batches)\n",
    "# Reshape each split into [batch_size, training_seq_len]\n",
    "batches = [np.resize(x, [batch_size, training_seq_len]) for x in batches]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Initialize all the variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Initialize all variables\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Training the model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch #1 of 10.\n",
      "Iteration: 10, Epoch: 1, Batch: 10 out of 182, Loss: 9.90\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-27-a0c2a8560b75>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m         temp_loss, state, _ = sess.run([lstm_model.cost, lstm_model.final_state, lstm_model.train_op],\n\u001b[1;32m---> 20\u001b[1;33m                                        feed_dict=training_dict)\n\u001b[0m\u001b[0;32m     21\u001b[0m         \u001b[0mtrain_loss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtemp_loss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    787\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 789\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    790\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    995\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 997\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    998\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1130\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1132\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1133\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1137\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1139\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1140\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[0;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Train model\n",
    "train_loss = []\n",
    "iteration_count = 1\n",
    "for epoch in range(epochs):\n",
    "    # Shuffle word indices\n",
    "    random.shuffle(batches)\n",
    "    # Create targets from shuffled batches\n",
    "    targets = [np.roll(x, -1, axis=1) for x in batches]\n",
    "    # Run a through one epoch\n",
    "    print('Starting Epoch #{} of {}.'.format(epoch+1, epochs))\n",
    "    # Reset initial LSTM state every epoch\n",
    "    state = sess.run(lstm_model.initial_state)\n",
    "    for ix, batch in enumerate(batches):\n",
    "        training_dict = {lstm_model.x_data: batch, lstm_model.y_output: targets[ix]}\n",
    "        c, h = lstm_model.initial_state\n",
    "        training_dict[c] = state.c\n",
    "        training_dict[h] = state.h\n",
    "        \n",
    "        temp_loss, state, _ = sess.run([lstm_model.cost, lstm_model.final_state, lstm_model.train_op],\n",
    "                                       feed_dict=training_dict)\n",
    "        train_loss.append(temp_loss)\n",
    "        \n",
    "        # Print status every 10 gens\n",
    "        if iteration_count % 10 == 0:\n",
    "            summary_nums = (iteration_count, epoch+1, ix+1, num_batches+1, temp_loss)\n",
    "            print('Iteration: {}, Epoch: {}, Batch: {} out of {}, Loss: {:.2f}'.format(*summary_nums))\n",
    "        \n",
    "        # Save the model and the vocab\n",
    "        if iteration_count % save_every == 0:\n",
    "            # Save model\n",
    "            model_file_name = os.path.join(full_model_dir, 'model')\n",
    "            saver.save(sess, model_file_name, global_step = iteration_count)\n",
    "            print('Model Saved To: {}'.format(model_file_name))\n",
    "            # Save vocabulary\n",
    "            dictionary_file = os.path.join(full_model_dir, 'vocab.pkl')\n",
    "            with open(dictionary_file, 'wb') as dict_file_conn:\n",
    "                pickle.dump([vocab2ix, ix2vocab], dict_file_conn)\n",
    "        \n",
    "        if iteration_count % eval_every == 0:\n",
    "            for sample in prime_texts:\n",
    "                print(test_lstm_model.sample(sess, ix2vocab, vocab2ix, num=10, prime_text=sample))\n",
    "                \n",
    "        iteration_count += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Here is a plot of the training loss across the iterations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEZCAYAAABiu9n+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeYFeX58PHvDQvs0qQIIkWkKYJY6GCB2AAbxqBoYmyo\n8UV/GI0FMEZM1GBLMCpqLAGNJhYsiCKE6GJUVEQQkRVURHqVhZWywO79/jHFOWV2z9Y5u3t/rutc\nnDPzzMxzhrNzz1NHVBVjjDEmmVpRZ8AYY0z6siBhjDEmlAUJY4wxoSxIGGOMCWVBwhhjTCgLEsYY\nY0JZkDDGGBPKgkQNIiLHi8gHIpIrIltE5H8i0ivqfFUGESkUkY5l2H6UiOSIyHYRWS8ib4hIg/LM\nYzoSkUEisjrqfJjoZESdAVM5RKQR8AbwG+AloC5wApAfZb4qUalHjYrIIOAu4DRVXSwiTYCzyi1n\n6c9G3NZgVpKoOQ4DVFVfVEe+qs5R1SVeAhG5XESWishWEZkpIocE1p3q3klvE5GHRCRbRC53190u\nIs8G0rZ379xruZ8bi8iTIrJORFaLyJ9ERNx1l7glmvtE5AcR+VZEhgb21VREnhaRtW6+XgmsO1NE\nFrp5el9EeiT74iIyFxBgsYjsEJHz3OVXisjXbqnqNRE5OOTc9QY+VNXFOCcxV1WfVdWd7n7qisj9\nIvK9W8qYLCL1Ase/yf3ua0TksmCpRkTe9c5j8HwEPncVkdnud8/x8u6u+4eIPCwiM9zvNU9EOgTW\ndw9su15ExrrLRUTGisg3IrJZRP7tBr4Scf9fnxGRTSLynYjcGljXyf2N5Lrr/xVY91cR2eiuWyQi\n3Up6bFN5LEjUHMuBAhGZIiJD4y8KInIOMBY4B2gB/A/4l7vuQOBlYDxwIPAtMDBu//F3m8HPzwB7\ngY7AscCpwBWB9X2BHKA5cB/wVGDdP4Es4AigJfBXN0893XRXAs2Ax4HpIlIn/our6iD3bQ9Vbayq\nL4nIScDdwAjgYGAV8O/4bV0fA0NEZIKIDBSRunHr7wU6A0e5/7YB/uDmcyhwA3Ay0AU4heLvzNXd\ntj4w2z0HBwIXApNF5IhA2guA24EmOP8vd7nbNgT+A7zlfr/OwH/dba4DzsYpSbYGtgGTi8lTMg8D\njYBDgcHAxSJymbvuT8AsVW0CtAUecvN1GnA80NldNxLYWopjm8qiqvaqIS/gcOBpnAviXuB1oIW7\n7i3gskDaWsBOoB3wa5w76eC+VgOXu+9vB54JrGsPFLj7OAjYA9QLrL8AeMd9fwmwPLAuCyjECQit\ngP1A4yTfZTJwR9yyr4ATQr57IdAx8PlJYGLgcwP3nBwSsv0Q93z9AOwAHgDEXfcj0CGQdgCwwn3/\nFHB3YF0X99x0dD+/653HwPl4z31/PjA3Lh+PAbe57/8B/D2wbhiw1H1/IbAg5LssBX4W+Hyw+91r\nJUk7CFiVZHkt9//18MCyqwL/r1PdvLaJ2+5n7v9TP+/82Su9X1aSqEFUdZmqXq6qhwBH4txFTnJX\ntwcedKt8fsC5u1Ocu+LWOEEhKNXGzEOAOsB6d9/bcC4eBwbSbAjkcbf7tiFOgPpBVXck2W974Hde\nft39tnXzmorWwPeB4+7E+c5tkiVW1VmqOlxVmwHDgUuBK0SkBVAfWBA4dzNxSkXecYLn6nucqq9U\ntAf6x33HX+IEXs+GwPtdOOcNnHPxbRH7fTWQ36XAvrj9FudAnP/XVYFl3/PT+bsZJ5B8IiJfeCUM\nVX0XpwTyCLBBRB5zSz0mTVmQqKFUdTkwBSdYgHMh+42qNnNfTVW1oap+BKzHudgHtQu834lzofQE\n6/ZX49xxNg/st4mqHpVCNlcDzUSkcci6u5Lk94UU9guwDudiCYA4PZWaA2uL29C90L2Dc+624Fyc\nuwfy0kRVD3CTryf2XLUntrop/ty1ivuO2XHfsbGqXpvC91uNU8WUzCpgWNx+G6jq+hT269mCE1ja\nB5a1xz1/qrpRVa9S1TbA1TjVZB3ddQ+ram+gO07p9qYSHNdUMgsSNYSIHC4iN4hIG/dzO5wqiXlu\nkseA8V4joogcICIj3HVvAt1E5BwRqS0i1xF717kIOFFE2onIAThtGwCo6gacevW/ikgjt9G0o4ic\nWFye3W1n4lxgmohIhoic4K5+ArhaRPq6+W0gIqdLeLfUDThtIp7ngctE5Ci3kflu4CNVXRW/oYic\nLSIjvXYc95iDgHmqqm5eJrmlCkSkjVv3DvAicKmIHOG2MfwhbveLgHNFJEtEOgOjAutmAIeJyEXu\nd68jIr1F5PDizp277UEiMkachvWG3rnCab+5W9yOCSLSQkTOLmJfIiL1gi9VLXS/213uvtsD1wPP\nuhuM8H5rQC5OdV+Bm/++IpIB7Ma5gShI4fuYiFiQqDnycOqBPxaRPOBDYDFwI4CqvgZMBP4tIrnu\nuqHuuq3AecA9OHeQnYAPvB2r6hzgBXeb+ThdbYMuxulyuxSnTv8lYu+Y4wXvtH+N0y7xFbARp9EV\nVV2A02j9sFtlshynPj/MBOAZt4plhKq+A9wGvIJz99sBp60kmW3usZaLyHachvh7VNVr6L4F+Ab4\nyD13s3F6k6Gqb+NU6b3j5vG/cfv+K84d+QacNoZ/+idB9UfgNDdf69zXRKAexXC3PRWngXqDe+zB\n7uoHcdpXZrvf50OczgNhWuOUlnbhXNh3uaWCMe6yFcB7wD9V9R/uNn1wfms7gNeAMar6PdAYJ6j+\nAHyH83u6v7jvY6LjNbxVzM5FngLOBDZ61Qvu3ekEnN4qfVT1swrLgKkwIvIu8KyqPh11XqoaESnE\n6d2zIuq8GFOcii5J/AOnV0jQF8DPgbkVfGxjjDFlVKEjrlX1fbeuMrhsGTiVnBV5bFPhbBRu6dm5\nM1WGTcthSkVVT4o6D1WVqtaOOg/GpMoaro0xxoRK65KEiFix3BhjSkFVy6VKvzJKEkL4CNNiv0TU\nQ9Kr0+v222+PPA/V5WXn0s5nOr/KU4UGCRF5HqcP9mEiskqcGTDPEWd++v7ADBGZWZF5MMYYU3oV\n3bvplyGrXqvI4xpjjCkf1nBdgwwePDjqLFQbdi7Ll53P9FWhI67LSkQ0nfNnjDHpSETQKtRwbYwx\npoqyIGGMMSaUBQljjDGhLEgYY4wJZUHCGGNMKAsSxhhjQlmQMMYYE8qChDHGmFAWJIwxxoSyIGGM\nMSaUBQljjDGhLEgYY4wJZUHCGGNMKAsSxhhjQlmQMMYYEyrtg4Q9T8IYY6KT9kFi3759UWfBGGNq\nrLQPEvn5+VFnwRhjaqy0DxJ79+6NOgvGGFNjpX2QsJKEMcZEJ+2DxO7du6POgjHG1FhpHyRyc3Oj\nzoIxxtRYaR8ktm3bFnUWjDGmxkr7IGElCWOMiU7aBwkrSRhjTHTSPkhYScIYY6JToUFCRJ4SkY0i\nsjiwrKmIzBaRZSIyS0QOKGof27dvr8gsGmOMKUJFlyT+AQyJWzYWmKOqhwPvAOOK2oEFCWOMiU6F\nBglVfR+Ib1QYDkx1308FzilqH3l5eRWQM2OMMamIok2ipapuBFDVDUCLohLbtBzGGBOdtG+4tiBh\njDHRyYjgmBtF5CBV3SgirYBNRSVevHgxEyZMAGDw4MEMHjy44nNojDFVSHZ2NtnZ2RWyb6noh/qI\nyKHAG6raw/18D/CDqt4jIrcATVV1bMi2etpppzFr1qwKzaMxxlQnIoKqSnnsq6K7wD4PfAgcJiKr\nROQyYCJwqogsA05xP4ey6iZjjIlOhZckykJEFOwRpsYYUxJVpiRRXixIGGNMNNI+SGRlZbFr166o\ns2GMMTVS2geJxo0bs2PHjqizYYwxNZIFCWOMMaEsSBhjjAllQcIYY0woCxLGGGNCWZAwxhgTKu2D\nRKNGjSxIGGNMRNI+SFhJwhhjolMlgoQ9eMgYY6JRJYKElSSMMSYaFiSMMcaEsiBhjDEmlAUJY4wx\noSxIGGOMCVUlgsTChQvZvXt31Fkxxpgap0oECYA//elPEefEGGNqnrQPEk2aNAEgNzc34pwYY0zN\nk/ZBol69egDs3bs34pwYY0zNkxF1BlJx1VVX0a5du6izYYwxNU7alyQAOnXqxNq1a6POhjHG1DhV\nIkgUFhby2GOPRZ0NY4ypcapEkDDGGBONKhEkrD3CGGOiIaoadR5CiYiqKqpKrVq1KCgooFatKhHX\njDEmMiKCqkp57KtKXHFFhIyMDAoKCqLOijHG1ChVIkgAZGRksH///qizYYwxNUqVCRK1a9e2IGGM\nMZUssiAhIteJyBfua0xx6a26yRhjKl8kQUJEugOjgN7AMcBZItKpqG2sJGGMMZUvqpLEEcBHqpqv\nqgXAXODnRW1gJQljjKl8UQWJJcCJItJUROoDpwNFDobYtGkTO3furJTMGWOMcUQywZ+qfiUi9wBz\ngDxgEZC0LmnChAn++z59+rB169bKyKIxxlQZ2dnZZGdnV8i+02IwnYjcBaxW1cfilquXPxFnXEg6\n5NcYY9JZtRhMJyIt3H8PwWmP+Fcq2+3bt68is2WMMSYgynES00RkCfA6MFpVt6ey0R133FGxuTLG\nGOOL7KFDqnpiabaz50oYY0zlqTIjrm+++WYA6wZrjDGVqMoEifHjxwPOA4iMMcZUjioTJBo1agRY\nw7UxxlSmKhMkvOdIbN+eUvu2McaYclBlgoQnLy8v6iwYY0yNEVnvptI477zzaNWqVdTZMMaYGqNK\nlSROPvlk9uzZE3U2jDGmxqhSQSIrK4vdu3dHnQ1jjKkxqlSQaNCggc0Ea4wxlciChDHGmFBVLkhs\n27aNuXPnRp0VY4ypEdJiqvAwwanCAXJycujWrRtgU4YbY0yY8pwqvEoFCVX1B9Wlc76NMSZK1eJ5\nEqXhPXjIGGNM5ahSQSJo5syZUWfBGGOqvSobJF577bWos2CMMdVelQ0SmZmZUWfBGGOqvSobJLKy\nsqLOgjHGVHtVNkhYScIYYypelQsS9erVA+wxpsYYUxmqXJBYunQpo0ePZvny5VFnxRhjqr0qNZjO\nM2jQIN577z2+//57DjnkkAhyZowx6avGDqbz5ObmArB48eKIc2KMMdVblQwSHhuBbYwxFSulICEi\nnUSknvt+sIiMEZEmFZu1cD169ABs/iZjjKloqZYkpgEFItIZ+DvQDni+wnJVjPvuuy+qQxtjTI2S\napAoVNX9wM+Bh1T1JuDgistW0WwgnTHGVI5Ug8Q+EbkQuASY4S6rUzFZKl6dOs6hf/e730WVBWOM\nqRFSDRKXAQOAu1T1OxHpAPyzLAcWketFZImILBaR50SkbqrbeqOtly9fztKlS8uSDWOMMUUo8TgJ\nEWkKtFPVUvc/FZHWwPtAV1XdKyIvAG+q6jNx6ZKOk3DXAdClSxcbWGeMMQGVPk5CRLJFpLGINAM+\nA54Qkb+U8di1gQYikgHUB9aVZOPJkycD8PXXX5cxG8YYY8KkWt10gKruAM4FnlHVfsAppT2oqq4D\nHgBWAWuBXFWdU5J9NGzYsLSHN8YYk6KMVNOJyMHA+cCtZT2oO8ZiONAe2A68LCK/VNWEbrUTJkzw\n3w8ePJjBgweX9fDGGFOtZGdnk52dXSH7TqlNQkTOA24DPlDV/yciHYH7VPUXpTqoyAhgiKpe6X7+\nNdBPVa+NSxfaJvHZZ5/Rq1cvwAbVGWNMUHm2SUQywZ+I9AWeAvoA+cA/gPmq+khcutAg4a4HYMGC\nBfTs2bPC8muMMVVJFA3XbUXkVRHZJCIbRWSaiLQt7UFV9RPgZWAh8DkgOCO5S8UrURhjjClfqVY3\n/QdnGo5n3UUXAb9S1VMrMG8plyTAqpyMMcZT6dVNIrJIVY8pbll5syBhjDElF8XzJLaIyEUiUtt9\nXQRsLY8MlEWTJj9NRGtBwhhjyl+qQeJynO6vG4D1wAicqToilZOT47+3Z14bY0z5SylIqOoqVT1b\nVVuoaktVPQdnYF2kWrVqxa9+9SsA9u/fH3FujDGm+inLk+luKLdclEF+fj4ADRo0YNmyZRHnxhhj\nqpeyBIm0eHbonXfeCUBhYSG33347n3zyScQ5MsaY6qPUg+lEZJWqHlLO+Yk/RpG9mwLpAGjRogWb\nN2+2RmxjTI1WaV1gRSQPSJZAgCxVTXXup1IpaZDwWJAwxtRk5RkkirzIq2qj8jiIMcaYqqksbRLG\nGGOqOQsSxhhjQlmQMMYYE6paBIlPP/2UN954I+psGGNMtVMtgkSvXr3o1q2b/3nVqlUR5sYYY6qP\nahEkAOrXr++/79GjR4Q5McaY6qPaBImsrCz//Y4dOyLMiTHGVB/VMkgAfP311xHlxBhjqo9InnGd\nqlRHXIMzyrpWrZ9iXv369dm5c2dFZc0YY9JWFA8dSnvxU3PUrl2b5s2b2xTixhhTBhU691KU8vLy\nANi7dy8ZGdX2axpjTIWqNiUJz/333x/zOfj0OmOMMSVT7YJEnz59Yj737t07opwYY0zVV63qYVTV\nnnVtjDHlqNqVJGrXrp2wbN++fRHkxBhjqr5qFyTgp0eael544YWIcmKMMVVbtQwSZ5xxBg0bNvQ/\nx3ePNcYYk5pqGSSOOeYYnnnmGf9zvXr1IsyNMcZUXdUySACccsop/vtnn33WnnttjDGlEMm0HCJy\nGPACoIAAHYHbVPVvcelSnpYj5Dj++y1bttC8efNS78sYY6qK8pyWI/K5m0SkFrAG6Keqq+PWlVuQ\nuPvuuzn88MPp27cvbdq0sXYKY0y1Vd2CxGk4pYgTkqwrtyAB0KRJE3Jzc5kyZQqXXHJJqfdrjDHp\nrLpN8DcS+FdlHCg3NxeA7777rjIOZ4wxVV6kI65FpA5wNjA2LM2ECRP894MHD2bw4MFlPq49lMgY\nU51kZ2eTnZ1dIfuOtLpJRM4GRqvq0JD15Vrd5LnuuuuYNGlSqfdrjDHprDpVN11IBVY1vfvuuxW1\na2OMqREiCxIikgWcArxSUccIq5oqLCy0cRPGGJOCyHs3FaWs1U0A06dPR0Q4++yzY5Yfc8wxLFy4\nsEz7NsaYdFStusAWpTyCRGBfCctUlWnTptG2bVv69etXLscxxpioWZAohQcffJDf/va3Mcvy8/P9\neZ3S+TwYY0xJVKeG60ozZsyYhGX33XdfBDkxxpiqo8aUJNz9ha5Lp/Owfv16Dj744KizYYypoqwk\nUQHuvfdeLr300qizAUDr1q3ZuHFj1NkwxhgLEp5bbrmFqVOnxizLz8/n2muvrbQ8DBo0iNGjRwOw\nd+/eSjuuMcaEqXHVTY888gjXXHNNaJr9+/f7z8n+9ttv6dy5c6VVRQWrw1atWkW7du0q5bjGmOrF\nqptKafTo0Zx55plFphk0aBAPPfQQABkZ0U1tpar079+fXbt2RZYHY4ypUSWJwH6LTaOqrFy5kg4d\nOjBjxgzOPPPMCi9RBPM1btw4/vznP7N8+XK2bNnCgAEDKvTYxpjqw0oSlWT//v0AzJ8/31+2ePFi\nvv322wo/9p///GcAPvvsMwYOHFjhxzPGmGRqZJBo1qwZ119/fZFpCgoK2LdvHwB33HGHv/zoo4/m\nZz/7WUrHWbJkScpPwAsrpezZsyel7Y0xpiLUyCCxefNmHnjggSLTfP/9936Q8Hz22WclOs4XX3yR\nctr4Y3kmTpwIOMXHLVu2lOj4xhhTVjUySNSqVavYO/wNGzYkXLh79eoFwOrVq5k8eTJPPvlk6Pbb\ntm1LuRvrxo0b6dSpU9J1X331lf/+8ssvT2l/xhhTXmpkkAh65JFHaNu2bcyyzp07s379+tC7e4Br\nrrmGK6+8EnCedLdixQoALrvsMh599FGaNWvG+PHjU8pDTk4Oa9asKTbdG2+8kdL+jDGmvNToILFo\n0SKuvPLKhFLFIYccwvr16/2G66KcfvrpjBo1yi8JTJkyxR8Qt27dupTyUVhYWMKcG2NM5ajRQeLo\no4+mTp061KrlnIbHH38ccBqt//jHPzJnzpxi9zFz5kzWr18PkFJQAdi0aRNDhgzxP8cHibJ2td29\ne3eZtjfGGE+NDhKezMxMAM444wwAjjzySDZv3hzTq6koH3zwAQB16tQJTbN161a/4XnJkiXMnj2b\npUuXIiLcfPPNADRu3LjYaqcLLrigyPXTp0+nfv36oeu9gLRz506/iswYY8JYkABmz57NkiVLaNKk\nCQAXXXRRue5/4cKF9O7dm+7du7N//36ysrIAWLt2rb8e4OCDD6ZNmzZF7uuFF17w37/44otkZ2fH\nrP/uu++K3L527drMnz+fsWPHhjaWG2OMx4IEThtE9+7dadCgAXPnzqVr167luv+ePXuycuVKNm3a\nRJ06dfj+++8ByMvLi0lXt27dlPbXqVMntm/fzsiRI/nZz37GuHHj2L17Nw0aNKCgoKDY7detW8fD\nDz9c8i9SSnl5eVYFZkwVZUEizoknnkiDBg38z157BcAVV1xRLse48MILAfjhhx9ilgfHVTz66KOc\neOKJSbdfsWJFTNqJEyfyww8/sGvXLr9dZO3atagq69at47333mPz5s1++uB3Sla9lZubyyeffFKK\nb5Zc586dGTZsWLntzxhTeSxIJBFsW2jTpo3fkBwMHmVx0EEHARQ5zuLqq6+mb9++oeu3bdsW89nL\n4y233AI4wW7WrFm0adOGQYMGcdNNN/lpd+7c6b9v164dixcvZvHixf6yW2+9lX79+vHRRx+l/J2W\nLVvmv1+5cqXfzgJOQ31OTk7K+yqNVEpQxpiSsyBRDG8sxNatW8utDt97oNDHH39cZLrDDz88dN3Z\nZ58d8zl+TMeuXbv8Kc8Bpk6dyllnnQX8VJLxHH300Zx66qkJ+yrJiPGuXbv6A/9efPHFhEfDqioD\nBgxg06ZN/rKPP/643Lr/ZmRklGhE+rx588rluMZUdxYkQjz99NMA3HbbbYAz31OXLl0qNQ+jRo1i\nz549jBgxoti0+fn5MZ83bNjAc889F7Ps7bffDt0+ePH2qqO8IFNQUOBXVzVt2pRRo0bFXNy9/T76\n6KOsWLEi6Wj2zZs389FHH3HDDTegqjz33HP079+fiRMn+ukXLFgQOhI+JycnofQUL76NJ8yaNWts\n0kRjUqWqaftysheNtWvXarLjAxX6Subqq68udruFCxeWy7H37NmTsOy+++7z33vL165dq6qqL774\nYkz6n//853r//fcroHv37k16zjZu3Oi/v/DCC/19/+tf/4rZTlV127ZtunXrVgV0xIgRof9fgH7z\nzTeqqrp169Yi/2+//fZbBXTp0qW6dOlSHThwYJHpg+68885i929M1Ny/qXK5DltJIkTr1q0T7s4B\nHnvssaTLoWwjp++4447QevX4qptkymPKjj179vgDCj2/+c1v2LBhAwD//ve//eWvv/46IsL5558f\nk37Xrl3k5uYCTm8tr4oryFsPJG33qFu3rl91NGDAAI488kh/30UpLCxk0qRJNG/ePDTNzp07/f+n\nbt268f777/Phhx/GpHn33Xf9sS/x+//973/PzJkzi8xHOlqzZg3ffPNN1NkwVVF5RZuKeBFhSaI4\n3l0vcXfdwc/PP/986F17jx49ii1BBOXk5FR4KeaZZ57Rxx57LKW0w4cPL/Vx5s+fn7Bs2bJlCef0\n7bff1jp16vifzzzzzJhz8sknn+ioUaP88x5MGwTomjVrdMeOHQroBx984Kd79tlnk6avX79+wv/B\nW2+9pYCee+65umvXrhL9Xkpiz549xabxSk2p6tChQ0q/M1M9YCWJ6MWPfB41alRCmuDI56OPPjpm\nndcgnipv7Mazzz5bou0uueSSEqVftWpVSulef/31Eu03qE+fPgnLDj/88IQG9aFDh8aUzoIN8XXq\n1GHMmDE89dRT/rJg433Lli3Zvn273824bdu2/uDFpUuX+um8gY3JBLsNA/6svq+88kpCya1169Ys\nWLDA/7xs2bKEqeWHDx9e7HiR6dOn+zMAhFm+fDmdO3fm008/LTJd0I8//phyWmOCLEiUQffu3f33\n8d1ZTz/9dE455RT/c7t27WIuThkZGV5pqUQGDhwYc9zipPrQI3AuUMGurOkgWAWXl5fH1q1b+eCD\nD9i/f7/fbXfGjBkJ223evJnVq1fHDBr0RqMHA3Qw8ATt2rWLli1b+p9zcnJYvny5/zkYXA488EDW\nr1/P3Llz/WXdunWjd+/eMedz+vTpxU67snLlyiLXe3kDJ9h6v6F58+axf/9+v+dcvNJUhRYWFqZ8\nU+JVSYJzI3PCCSeU+HgmPWVEdWAROQB4EjgSKAQuV9Wi+4SmmWRPnlNVRIQJEybElCRq1arFEUcc\nwYoVK+jYsWPMgLZUeReEWbNmJUxvXtw2qXj55ZdLnKfK9M4779C1a9eYXldA0nYPcHp8BUsXp59+\nekKa4s5P9+7defPNN+nWrVvM8qysLH788UdWrVrF1q1bgZ8u3uvXr/cvyl27dkVV/UfeliRohwkG\nzsLCQr788suY3lqluflIZuXKlVx88cX8+te/LjLdnj17OPjgg/3jvvrqq7z//vvlkgcTvShLEg8C\nb6nqEcDRQMWOtqogqpow++u+ffvo06cPIsKvfvUr4KdupR06dADKdrFo06ZNQoNxmHr16pX6OOlo\ny5YtftVJWAcCz5lnnsmdd95ZZBrvwrZ9+/aYQYaepUuXxlQjeUSE8ePHx5Tqdu7cyc6dOznqqKMS\n0nfu3Nnf7ocffvCnZinOmjVrYhrRlyxZEhMkCgoKEqoy433zzTclDhwi4s9unMyVV17pf3cvP8mO\n8dVXX6X0W58xY0ZMF/Nt27YxderUlLs179ixI+n/nym7SIKEiDQCTlDVfwCo6n5V3RFFXspDfJVF\nRsZPBbTf/va3ABx77LHleswWLVqklK64x7RWRaleDIJVIGG8doYTTzyRhg0bJm2TSVZVs2/fvoT2\nhYkTJ9KwYcNiB/WdddZZHHrooeTl5dGuXTvuuusuPv/8c0466aSE6enHjBnD8ccf75cUevToEZOm\nqJHm+fn5FBQU0KVLl5gpYDZu3OhXvW3duhUR8Us6QWHnT1V58skn/epTLw/xN0vXXHMNX375ZWj+\nPCLCm2+rkaaUAAAXd0lEQVS+GdP76t577+XSSy/l/PPPp2fPnsUGi8MPPzxmQGh5e+SRR2JmJajo\nGQTSSVQliY7AFhH5h4h8JiJ/F5HwFsQqrHfv3uzfv98flOfx7q4mTZpUqv16kwHGdyEdMWIE27dv\n56677gKgYcOGfoOtSeSVRrwLQPv27RPSeOcyaNiwYSlXGU6ePNl/P3r0aL/LrTc1/Ny5c7nxxht5\n9913YxrEd+7cyauvvgo4bQ7eQ6yCXYGTPcPEu6PPysryp2kJOvnkk+nYsSOq6geCzp07+/v1ppAP\nKwFMmzYt5rOXh9WrV7Nx40b/+JMnT/bfL1++vMhpaLxgPXfuXAYNGuQvX7duHQsXLmT16tWh24IT\n0L744guuv/76lKf4L0qbNm2YMGGC//naa6/lkUce8T9369Yt6VT7eXl5vPnmm2U+fjqJKkhkAD2B\nR1S1J7ALGBtRXipc7dq1E/7gvM8XXXSR/xyLkvDml4q/UDVo0IDGjRvTs2dPf1nr1q1LvP+aorgq\nK4DPP/886fKiHm8b9Ic//MF/P3v27IT1mZmZSR9wFf+MdG+CyWDJJllJYtKkSTzwwAOoaszdL8D7\n77/v393XqlUrZubhBg0aMGLECH88jDctfXw+gqWttWvX+kGiU6dOtGrVKiatFyTuvPPOpD36vHPo\nlb6nT5/Oe++9lzAFfmFhIQUFBf4Nz6uvvhpzXsH5v5w0aRJ//OMf/VLepEmT/Kq+ZLKzs/3/kx9/\n/JEdO5wKjXXr1iX0Touvuk3223n88cc588wzQ49XFUXVcL0GWK2qXh++l4HEWx6IieaDBw9m8ODB\nFZ23SuEFiebNmyftnVMcL0gEq7YAv0fOkCFDQksQRx11FAsWLAh9SNK8efMYMGAA4BTjn3rqKY4/\n/vjQvNx999288sorfPrppyxdujShkTedlWWAWXHP7vAUV1WSbCCkiCRcKL2LcXFB4oYbbvDf/+c/\n/4lZN3To0JjP8b+BadOm+VWjL774IuBcHL2L/Zw5c/jd737npw92LU6mqDmytmzZEnpBjS8h79u3\nj7///e+MHj2aiRMn8tJLL7FgwQLWrl3L1Vdf7aepV68e+fn51K9fn5UrVzJ79uykVWmeoUOHkp+f\nj6py8skns3r1ar/E1rRp05i08VP5J2uDif97rCzZ2dkJv5fyEsk3UtWNIrJaRA5T1eXAycDSZGmD\nQaI6KWsvF++P+5hjjvGXPf300/74DREJLUF07do14cdcu3ZtCgoKmDZtGv379ycrK4vdu3fTpEkT\nP2B44gNB06ZNmTVrFs2bNw8NPN7+ijJp0iS/DaeypDKaPUyqf5Txd+Kpip9g0bvgB7ullvTvI749\n5+67705Ik2xMRWFhIWvWrEla7+/dfXuCF08vCMdfUF955RU++uijhEku49N5pZ5gyXjs2LH+56ef\nfprGjRv76xo3buyPb8nJySn2kcLBgOtNj+9t07t375i08SUJVWXcuHFMmDDBXxfWpbqixd9Al0eV\nmyfK3k1jgOdEZBFO76bEX2s11rFjxzJtf9xxx5GVlYWIsGHDBjIyMhg2bFjo4LAjjjjCf58sQHnV\nVueee27MZxFJqNKKbzS/8sor/eOG1dOncocVX1UBsfX58Y2gU6ZMKfZJflVZWHuVd6cLlPnhUU88\n8UTCsokTJyYs+81vfpO0vcZbFxQcaBk2OPMXv/hFTID2fpPxQSKsYT6YLhgIggFj2LBhCUHiyy+/\nRFW57bbb+N///pe0ynDq1KkA/mSVwTaekSNH+v8vRx55JBMnTowZPxNVSaJCldfQ7Yp4UU2nESgs\nLKz0Y+7evdufimLkyJGqGjuFyMCBA7V79+5++kaNGimgAwYMSEi7fft2veeee2KmwCgsLFRAv/vu\nOwW0b9++MdtMmDBBgZjJCkeOHBmT5oUXXkiYriM4tUmyyQdbt27tf65bt67Wq1cvYR/ehH72iv7l\niV8+fvx4BfSqq65KaT+HHnqo//6yyy7z3x977LEx6U488cSE406fPl3BmV7FS7d//37/fVZWVtJj\nehNdHnHEEUnXz5gxQx9//PGY40XFzYNNy1FVlceAqpKqW7duwrQXP/74I8cffzx9+vThnXfeiZnm\nIViSCProo49o3LhxzEOFgum8O7MhQ4b41VRr165l/PjxQGyJ5uKLL066D3DuNCG2zrxu3boJpYlx\n48b54xLatm3Lnj17/DpqT3GltrAnABYllenbi1JeTzmsLrw7/lSnewmOTA9W5wVLEvBTSWT48OH+\nXb7XRhSsGgqWOMKqRb2qurD1H374YbUsSViQqCFq1arF888/D/x0MW7QoIE/42m9evVi5gzy0gSr\nj5YtW0a/fv2KPM7evXtZv349t99+u7+sdevW1KlTh7179/J///d/fvE9rMcX/FRsDx5fRBIaxa+9\n9lq/ysULKMH5qlJ5mmCwoTco+IyNeKkE+kaNGoWuSzbXV00gIjzzzDMJy/fs2QMQOq1IUYLPTTng\ngANi1nkX9unTp/sBw/s3GCRSebKhNzYiLEjEP+irurAgUQMF6/EzMjKSNjb/5S9/AX66GH722Wcc\ndthhMWkmT54c06Xz+OOPp127drRq1SrpH0udOnUSLq5h3X/vuOMOpkyZEtrGcfLJJ/vvvYFmXu+T\nYC+UZHXO8WNWgr13gg488MCkx4bwIOEFqH79+oU24kPREwtWd8kmnSyqB1KqkrXJJRvY6DX8e4EJ\nEufMip/AE34aP+JNwxJv0qRJCZNCVgvlVW9VES/SoG6vutm0aZPm5+enlBa3Tre0+vfvH1o/C+jM\nmTP1ggsu8Ot0X3755YS669dee00B/eKLL2K2PemkkxL216tXL1V12l9uvfVWBVRE/PXBfQc///e/\n/41Zduihh+qwYcNUVbVWrVr+8uD7YL6DL69Ofdu2bXrggQeG1ql/9dVX/ntvuvKi9uu9Up3KvTSv\nli1bVti+K+M1dOhQbdCgQbnsK9jOUZKX10ZWUFBQ6r+b8gDWJmFKqUWLFgn9vYtS1N1wWTzxxBMM\nHjyYRx99NGb5WWedFVO68EoS3oOHPBp35//+++/7/fozMzO58847Oemkk2KeE37NNdf4VUjeneuw\nYcPo379/zL4uuugi3nrrLSC2i2TwmGElHO98ZWZmJpSmgqUfb90DDzwQM/eSVyUY7/rrrwdiR4Tv\n2rUrYUbg4EzDxeU13rBhw1JKl65yc3PLbf6m4OScJeH1PEt1oGVVYEHChFqwYEGJn1+RqiuuuILM\nzEyaNGkS01Yyffr0mMGFqTbyH3fccQkN1DNnzmT+/Pn+54YNG/rdd720f/vb3xIuCME/8Ouuuy7p\n8cLy5QWVzMzMhIvznDlz/Cq7hg0bAk7g6NGjh1/dEbbfgoICPv30U0477TR/mdcFOihZl+BUuwl7\nQTCV+vl4PXr0SFhW2gttaSV7ymFplfXmKJWR/FWFBQkTqmfPnhx88MGl3n7s2LHceuutxaa78MIL\n6dq1a8yAKc8pp5wS82Chkqhbt65/MYbESRZVNWbKhkWLFgGxcyNNmjTJ/xwsSQQvzt6o4bfeeov7\n77/fH0CWrN3Bqwdv1aoVo0aN8mc+bdmyZZGdAgoKCujVq1expYJkxyzpsyS8Y4QNxozvQXTzzTf7\nA/+C5+jpp59OGPGdikMPPRSIfVxuZQubwC/s/McPOLUgYUwKhg8fXuxU3Z6cnBz/4hCUmZnJ5Zdf\nHrPs4osvLnHvIFVl5MiRRabxqn3i77yzsrIYPnw4xx13HABvv/22/2yKiRMn+nedw4YNo379+nTq\n1AlwnpUdX/0TnI31ySef9O+2s7Kykt4Je4288Xf3waqrIC8v27dv95elGiTiq/AyMjJYsGCBX9Xn\n9Tzyqsq8QDl+/PjQXj3eZH2pPEzJ07dvX4CEm4bXXnst5vOYMWP8wB5UHg/Oip+SwxP21MD4c2dB\nwpgITZ061X9OR3nLy8vjxhtvTFj+2muvMWrUKLp168aQIUO48MILUVVuueWW0GqVQw45JGZcCDhj\nU1KpypgyZQo5OTlMmTIFiC25tGzZkvPOOy902507d9K4cWN/zEGqz2SIDybNmzenZ8+efjfiZs2a\nAT/dTXuTEmZmZibtHRYcrR9fwvGCyoMPPpiwnVd1Vbt27Zg2q/i7+AcffJAjjzySSy65JKYbbHwv\nvGCb03vvvRez7qqrrko4PoT3bAt+j+bNm/vvg+MsWrRoYUHCmOqqYcOGoXfFl112WdLnI5xzzjkx\nc2jFi28QTuVZIIcddpj/XPN58+bFTFe+ceNGfyqMZG0YXtDyqtpef/31hIv4Qw89lLBdMM2KFSt4\n++23AaearkGDBv7YFS+d96zyunXrhpZWwoJE2NME+/Tpw+9//3v/uwWrcZLdxdeuXZspU6YkBIYg\n7zwUFBQkPFY1vtQ4ZMiQpAHP209wMsYxY8b4770gUb9+fZo3b25BwhjzkxEjRrBw4cLQ9cEBgNdd\nd13MQMMwwQtV//79Q6s/Lr30Uv/OO77Nxbtwn3TSSQnbXXPNNUBsXXowOHbo0MGfUfjhhx+OGRvg\nXRC9Se2K6lzgrcvMzOTGG2/0q668vBW1bfzFuqiG8OBdfTxvgF2y9oT4wOMd0/v37LPPBpwxQHl5\neTHBKDi62qsObN68uT8TbXVhQcKYChYc9T1p0qTQKo6gsLvZeDfccIP/zIj4huawCfngp4tzsOqr\nc+fOSdtFateuHTMDqnexrVu3btJuwY899ljCd6lTpw733XefX4IJCxJFfS4qSCSrHho3bhxDhgwp\nsst3WBuDJ74tKHgekk3ZLiLVLkhUv4lGjEkzY8eO9WfXTcW0adOKnf4kmfg75S5dusRcxGfMmMGc\nOXNixqF4F9C33nqLgQMHJkxrkcxTTz3lP6s92fHjx50EeRfW4AX2pZde4rTTTks4dny1X7du3eje\nvbs/k2tQfADZuHEjTZs2pU6dOklHT3uKCxJed2jveMH0we/QpUsXli5diohQt27dUk8Pn46sJGFM\nBcvKyooZMFecc889t1QTxRV3gT/jjDP461//yimnnOIvu+CCCxgxYgTDhg1LKUDccsstDBkyJKFq\nq1WrVn7g8L5rsiog7447WJIYMWKE363WKzksXLiQ9u3b++0y4NzF//KXv0yaLy+gzJw5E3Aa971S\nUlG9u7zG+PhHq3pBIX6q8eD/i9d54vrrr/e762ZkZFhJwhiTfr766qtSjWkZMGBAiboTJ3vWBMCS\nJUtiSjKbNm1K2kDvXZRHjx7tN4wHed2HvY4AdevW5Z577vGf1X3OOeck7TzgSdbpIL7KqFmzZn5X\nZK+E520Xv723bbLqv06dOtGjRw+GDx9OZmYmixYtonHjxqhqSp0TqgoLEsZUA8HpR0qivKa2ji81\nhF0ke/XqRW5url9qCbY5bNu2LelgwDFjxvgjzbt16xbT3TVesiARX5LYunUrmZmZ5OfnU6tWLcaN\nG8fpp5/Op59+Stu2bWPS/v73v2fu3LmhbUTBZ4iXpLRYlViQMKYGi+KO1wsQDz30UMwAxyZNmiRN\nn5mZWWQX46DgCHrPUUcdlTAQr0ePHv48T94jXL3eXAA33XQTvXr1iqma88yfP9/v/lsTSKq9KKLg\nzuAZdTaMMVWYqlJQUBBTatq9ezeqmtL8UiJC//79mTdvHgCbN2+mZcuWKfdAi4L72NVyebqZlSSM\nMdWaiCRUq5XleR4tWrRI6wBR3qx3kzHGFEFEYnpZ1TRWkjDGmCLk5eWV6Bks1Y21SRhjTDVTnm0S\nVt1kjDEmlAUJY4wxoSxIGGOMCWVBwhhjTCgLEsYYY0JZkDDGGBMqsnESIrIS2A4UAvtUtW9UeTHG\nGJNclCWJQmCwqh5rAaJyZGdnR52FasPOZfmy85m+ogwSEvHxaxz7Qyw/di7Ll53P9BXlRVqBWSIy\nX0SujDAfxhhjQkQ5d9NAVd0gIi2A/4hIjqq+H2F+jDHGxEmLuZtE5HYgT1X/Erc8+swZY0wVVKWf\nJyEi9YFaqvqjiDQATgPuiE9XXl/SGGNM6URV3XQQ8KpbUsgAnlPV2RHlxRhjTIi0qG4yxhiTntKy\nC6qIDBWRr0RkuYjcEnV+qgoRWSkin4vIQhH5xF3WVERmi8gyEZklIgcE0v9NRL4WkUUiktqT5qsx\nEXlKRDaKyOLAshKfPxG5xP3tLhORiyv7e6SLkPN5u4isEZHP3NfQwLpx7vnMEZHTAstr/PVARNqK\nyDsislREvhCRMe7yiv99qmpavXAC1zdAe6AOsAjoGnW+qsILWAE0jVt2D3Cz+/4WYKL7fhjwpvu+\nH/BR1PmP+gUcDxwDLC7t+QOaAt8CBwBNvPdRf7c0Op+3AzckSXsEsBCn+vlQ9xrgjaWq8dcDoBVw\njPu+IbAM6FoZv890LEn0Bb5W1e9VdR/wb2B4xHmqKpINUBwOTHXfT+WnczkceAZAVT8GDhCRgyoj\nk+lKnS7Y2+IWl/T8DQFmq+p2Vc0FZgNDqYFCzic4v9N4w4F/q+p+VV0JfI1zLbDrAaCqG1R1kfv+\nRyAHaEsl/D7TMUi0AVYHPq9xl5niBQcoXuEuO0hVN4LzQwNausvjz/Na7Dwn0zLF8+f9Tu28Fu8a\ntwrkyUD1SNh5s+tBHBE5FKeE9hGp/32X+veZjkEi2V2Gta6nZqCq9gZOx/lDPIHwc2fnuWziz5/g\nnD87r0WbDHRS1WOADcAD7vKw82bnM0BEGgIvA9e5JYpU/75L/ftMxyCxBjgk8LktsC6ivFQp7p0E\nqroZeA2nqL7Rq0YSkVbAJjf5GqBdYHM7z8mV9PzZ77cIqrpZ3cpx4Amc3yjY+SyWiGTgBIhnVfV1\nd3GF/z7TMUjMBzqLSHsRqQtcAEyPOE9pT0Tqu3cZBAYofoFz7i51k10KeD+u6cDFbvr+QK5XbK3h\nhNi7rZKev1nAqSJygIg0BU51l9VUMefTvZB5zgWWuO+nAxeISF0R6QB0Bj7BrgdBTwNLVfXBwLKK\n/31G3Wof0pI/FKf1/mtgbNT5qQovoANOz4+FOMFhrLu8GTDHPZ//AZoEtnkYp+fI50DPqL9D1C/g\neZy7qnxgFXAZTm+QEp0/94/1a2A5cHHU3yvNzuczwGL3t/oaTp26l36cez5zgNMCy2v89QA4DigI\n/I1/5p6XEv99l/T3aYPpjDHGhErH6iZjjDFpwoKEMcaYUBYkjDHGhLIgYYwxJpQFCWOMMaEsSBhj\njAllQcJUayKS5/7bXkQuLOd9j4v7bM9oN9WOBQlT3XkDgToAvyzJhiJS3N/H+JgDqR5fkv0bUxVY\nkDA1xZ+B490H3VwnIrVE5F4R+didkfRKABEZJCLvicjrwFJ32avuzLpfeLPrisifgSx3f8+6y/K8\ng4nIfW76z0Xk/MC+3xWRl9wH6zwbSD9RRL5083JvpZ0VY4oR1TOujalsY4HfqerZAG5QyFXVfu6c\nQB+IiPec9WOB7qq6yv18marmikgmMF9EpqnqOBG5RlV7Bo6h7r5/ARylqj1EpKW7zVw3zTFAN5wZ\nUD8QkYE401Cco6pd3e0bV9RJMKakrCRhaqrTgItFZCHwMc4cOF3cdZ8EAgTAb0VkEc78/W0D6cIc\nB/wLQFU3AdlAn8C+16szH84inKew7QB2i8gTIvJzYHcZv5sx5caChKmpBPg/VT3WfXVS1Tnuup1+\nIpFBwElAP3WegbAIyAzsI2zfYZ/zA+8LgAxVLcCZMnsacCbwdmm+kDEVwYKEqe68C3Qe0CiwfBYw\n2p2jHxHpIiL1k2x/ALBNVfNFpCvQP7Bur7d93LHeA0a67R4tgBNwpr1OnkHnuE1U9W3gBuCo1L+e\nMRXL2iRMdef1bloMFLjVS1NU9UH3MZCfiYjgPKzlnCTbvw1cLSJf4kzHPC+w7u/AYhFZoKq/9o6l\nqq+6c/h/DhQCN6nqJhE5IiRvjYHX3TYPgOtL/3WNKV82VbgxxphQVt1kjDEmlAUJY4wxoSxIGGOM\nCWVBwhhjTCgLEsYYY0JZkDDGGBPKgoQxxphQFiSMMcaE+v9U+0VsfO5awgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4ac44402b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot loss over time\n",
    "plt.plot(train_loss, 'k-')\n",
    "plt.title('Sequence to Sequence Loss')\n",
    "plt.xlabel('Iterations')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
